# -*- coding: utf-8 -*-
import scrapy,sys,os,json,time
from pymongo import MongoClient
reload(sys).setdefaultencoding('UTF-8')
import logging
from pyquery import PyQuery as pq
import string
import random
import requests


class bkApiSpider(scrapy.Spider):
        name = "bkApi"
        allowed_domains = ["tmkoo.com"]
        '''start_urls = [
                "http://www.crca.com.cn/list.aspx?cid=20&page=1",
                ]'''
        
        def __init__(self):
            self.apiKeyView = 'TEST_APISTORE'
            self.apiPasswordView = '0HTCv5hvPH'
            self.apiKey = 'app18:59:36:d1'
            self.apiPassword = 'w3IarsdObP'
            self.results = {}
            self.saveCheckList = []
            self.regNoList = []
            self.insertArray = []
            #self.getApiKeyPassword()
            self.count = 0
            self.saveNum = 1
            self.maxNumList = []
            self.regNo = '1'
            client =  MongoClient("192.168.1.88", 27017)
            db = client['shangbiao']
            self.bk = db['bkScrapy']
            self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Referer": "http://www.tmkoo.com/searchmore/zch.php",
            "Accept-Language": "zh-cn",
            "User-Agent": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C; .NET4.0E)",
            "Host": "www.tmkoo.com",
            "Content-Type":"application/x-www-form-urlencoded",
            "Pragma": "no-cache",
            "Connection":"keep-alive",
            "Cookie": "JSESSIONID=657F80F8821239BB779ABF7B9F72BAEC;" ,
            }

            '''self.APIkey = 'UYT856h09TQurw8rsW'
            self.APIpassword = 'Tdwh58dkP04w'
            connection = pymongo.MongoClient('localhost',27017)
            db = connection['shangbiao']
            self.collection = db['shangdun']'''

        def getApiKeyPassword(self):
            mobile = ''.join([random.choice(string.digits) for _ in range(11)])
            url = 'http://api.tmkoo.com/app-reg.php?mobile=%s' %(mobile)
            res = requests.get(url)
            res.encode = 'utf-8'
            if res.status_code == 200 :
                info = json.loads(res.text)
                self.apiKey = info['apiKey']
                self.apiPassword = info['apiPassword'];
                print "Pwd generate successfully"
                return info
            else :
                print 'pwd generate fiald'

        def store(self,filename,data):
            with open(filename, 'w') as f:  
                json.dump(data,f)

        def saveCheck(self,item):
            self.saveCheckList.append(item)
            if len(self.saveCheckList)>100000:
                self.store('r'+self.regNo+'.json',self.saveCheckList)
                del self.saveCheckList[:]

        def closed(self,reason):
            #self.bk.insert(self.insertArray)
            self.maxNumList.sort()
            Num = self.maxNumList[-1:]
            self.store('rleft_%s.json' %(Num),self.saveCheckList)
            print "vailed Num is %s,last No is %s" %(self.count,Num)
            

        def start_requests(self):#19000000
            for x in xrange(3677890,4000000):
                url = 'http://api.tmkoo.com/app-search.php?keyword=%s&apiKey=%s&apiPassword=%s&pageSize=15&condition=2' %(x,self.apiKey,self.apiPassword)
                request = scrapy.Request(url,meta={'regNo':str(x)},callback=self.parse)
                #print "start checking regNo.%s" %(x)
                yield request

        def parsePwd(self,response):
            if response.status == 200:
                info = json.loads(response.body)
                self.apiKey = info['apiKey']
                self.apiPassword = info['apiPassword'];
            else :
                print 'pwd generate fiald'

        def parse(self,response):
            self.regNo = regNo = response.meta['regNo']
            if response.status == 200:
                info = json.loads(response.body)
                if len(info['msg'])>0:
                    print "parse_Search regNo.%s error msg is :%s" %(regNo,info['msg'])
                    self.getApiKeyPassword()
                    url = 'http://api.tmkoo.com/app-search.php?keyword=%s&apiKey=%s&apiPassword=%s&pageSize=15&condition=2' %(regNo,self.apiKey,self.apiPassword)
                    request = scrapy.Request(url,meta={'regNo':regNo},callback=self.parse)
                    yield request
                else:
                    if len(info['results'])>0:
                        intCls = info['results'][0]['intCls']
                        item={'regNo':regNo,'intCls':intCls}
                        self.saveCheck(item)
                        #print "regNo.%s Existed,intCls is %s,start request info" %(regNo,intCls)
                        url = 'http://api.tmkoo.com/app-info.php?apiKey=%s&apiPassword=%s&regNo=%s&intCls=%s' %(self.apiKey,self.apiPassword,regNo,intCls)
                        request = scrapy.Request(url,callback=self.parse_Info)
                        request.meta['item'] = item
                        yield request
                    else:
                        print "regNo:%s Is Not Existed" %(regNo)

        def parse_Info(self,response):
            item = response.meta['item']
            regNo = item['regNo']
            intCls = item['intCls']
            info = json.loads(response.body)
            if response.status == 200:
                if len(info['msg'])>0 :
                    print "parse_Info regNo.%s error msg is :%s" %(regNo,info['msg'])
                    self.getApiKeyPassword()
                    infoUrl = 'http://api.tmkoo.com/app-info.php?apiKey=%s&apiPassword=%s&regNo=%s&intCls=%s' %(self.apiKey,self.apiPassword,regNo,intCls)
                    item={'regNo':regNo,'intCls':intCls}
                    request = scrapy.Request(infoUrl,meta=item,callback=self.parse_Info)
                    yield request
                else:
                    #print "regNo.%s info request successfully,start request goodsInfo" %(regNo)
                    info['flow'] = json.dumps(info['flow'])
                    goodsUrl = 'http://api.tmkoo.com/app-goods.php?apiKey=%s&apiPassword=%s&regNo=%s&intCls=%s' %(self.apiKey,self.apiPassword,regNo,intCls)
                    request = scrapy.Request(goodsUrl,meta={'regNo':regNo,'intCls':intCls,'info':info},callback=self.parseContent)
                    yield request
            else:
                print "failNo.%s" %(regNo)
                with open('fScrapy.json', 'a') as f:  
                    f.write(regNo+"\n");

        def parseContent(self,response):
            goods = json.loads(response.body)
            regNo = response.meta['regNo']
            intCls = response.meta['intCls']
            info = response.meta['info']
            info['goods'] = json.dumps(goods['goods'])
            self.count = self.count+1
            self.maxNumList.append(regNo)
            print 'regNo:%s,tmName:%s,count is %d,request successfully' %(regNo,info['tmName'],self.count)
            self.bk.insert(info)
            '''if self.count>50*self.saveNum:
                self.bk.insert(self.insertArray)
                self.saveNum = self.saveNum +1
                print "==============================Records:%s created successfully========================" %(self.saveNum*50)
                del self.insertArray[:]
            else:
                self.insertArray.append(info)'''
            if len(self.maxNumList)>1000:
                self.maxNumList.sort()
                Num = self.maxNumList[-1:]
                print "checking is until Num.%s" %(Num)
                del self.maxNumList[:]
